Before you start

Set my seed

# Any number can be chose
set.seed(567890)

Goals for this file

  1. Use raw fastq and generate the quality plots to asses the quality of reads

  2. Filter and trim out bad sequences and bases from our sequencing files

  3. Write out fastq files with high quality sequences

  4. Evaluate the quality from our filter and trim.

  5. Infer errors on forward and reverse reads individually

  6. Identified ASVs on forward and reverse reads separately using the error model.

  7. Merge forward and reverse ASVs into “contigous ASVs”.

  8. Generate ASV count table. (otu_table input for phyloseq.).

Output that we need:

  1. ASV count table: otu_table

  2. Taxonomy table tax_table

  3. Sample information: sample_table track the reads lost throughout DADA2 workflow.

Load Libraries

#Effecient package loading with pacman
pacman::p_load(tidyverse, devtools, dada2, phyloseq, patchwork, DT,
               install = FALSE)

Load Data

#Set the raw fastq path to the raw sequencing files
#Path to the fastq files
raw_fastqs_path <- "data/01_DADA2/05_reverse_fastq/"
raw_fastqs_path
## [1] "data/01_DADA2/05_reverse_fastq/"
#What files are in this path (Intuition check)
list.files(raw_fastqs_path)
##  [1] "SRR17060816_2.fastq.gz" "SRR17060817_2.fastq.gz" "SRR17060818_2.fastq.gz"
##  [4] "SRR17060819_2.fastq.gz" "SRR17060820_2.fastq.gz" "SRR17060821_2.fastq.gz"
##  [7] "SRR17060822_2.fastq.gz" "SRR17060823_2.fastq.gz" "SRR17060824_2.fastq.gz"
## [10] "SRR17060825_2.fastq.gz" "SRR17060826_2.fastq.gz" "SRR17060827_2.fastq.gz"
## [13] "SRR17060828_2.fastq.gz" "SRR17060829_2.fastq.gz" "SRR17060830_2.fastq.gz"
## [16] "SRR17060831_2.fastq.gz" "SRR17060832_2.fastq.gz" "SRR17060833_2.fastq.gz"
## [19] "SRR17060834_2.fastq.gz" "SRR17060835_2.fastq.gz" "SRR17060836_2.fastq.gz"
## [22] "SRR17060837_2.fastq.gz" "SRR17060838_2.fastq.gz" "SRR17060839_2.fastq.gz"
## [25] "SRR17060840_2.fastq.gz" "SRR17060841_2.fastq.gz" "SRR17060842_2.fastq.gz"
## [28] "SRR17060843_2.fastq.gz" "SRR17060844_2.fastq.gz" "SRR17060845_2.fastq.gz"
## [31] "SRR17060846_2.fastq.gz" "SRR17060847_2.fastq.gz"
#How many files are there?
str(list.files(raw_fastqs_path))
##  chr [1:32] "SRR17060816_2.fastq.gz" "SRR17060817_2.fastq.gz" ...
#Create a vector of reverse reads
reverse_reads <-list.files(raw_fastqs_path, pattern = "_2.fastq.gz", full.names = TRUE)
#Intuition check
head(reverse_reads)
## [1] "data/01_DADA2/05_reverse_fastq//SRR17060816_2.fastq.gz"
## [2] "data/01_DADA2/05_reverse_fastq//SRR17060817_2.fastq.gz"
## [3] "data/01_DADA2/05_reverse_fastq//SRR17060818_2.fastq.gz"
## [4] "data/01_DADA2/05_reverse_fastq//SRR17060819_2.fastq.gz"
## [5] "data/01_DADA2/05_reverse_fastq//SRR17060820_2.fastq.gz"
## [6] "data/01_DADA2/05_reverse_fastq//SRR17060821_2.fastq.gz"

Raw Quality plots

# Randomly select 12 samples from dataset to evaluate 
# Selecting 12 is typically better than 2 (like we did in class for efficiency)
random_samples <- sample(1:length(reverse_reads), size = 12)
random_samples
##  [1] 16 22 15  1 14  6 30 27 11 13 23 32
# Calculate and plot quality of these samples
reverse_filteredQual_plot_12 <- plotQualityProfile(reverse_reads[random_samples]) + 
  labs(title = "Reverse Read: Raw Quality")

# Show plots
reverse_filteredQual_plot_12

Aggregated Raw Quality Plots

# reverse reads
reverse_preQC_plot <- 
  plotQualityProfile(reverse_reads, aggregate = TRUE) + 
  labs(title = "Reverse Pre-QC")

# Show the plot
reverse_preQC_plot

Prepare a placeholder for filtered reads

# vector of our samples, extract the sample information from our file
samples <- sapply(strsplit(basename(reverse_reads), "_"), `[`,1)
#Intuition check
head(samples)
## [1] "SRR17060816" "SRR17060817" "SRR17060818" "SRR17060819" "SRR17060820"
## [6] "SRR17060821"
#place filtered reads into filtered_fastqs_path
filtered_fastqs_path <- "data/01_DADA2/06_filtered_fastqs_reverse"
filtered_fastqs_path
## [1] "data/01_DADA2/06_filtered_fastqs_reverse"
filtered_reverse_reads <- file.path(filtered_fastqs_path, paste0(samples,
                                                  "_R2_filtered.fastq.gz"))
#Intuition check
length(filtered_reverse_reads)
## [1] 32

Filter and Trim Reads

Parameters of filter and trim DEPEND ON THE DATASET

  • maxN = number of N bases. Remove all Ns from the data.
  • maxEE = quality filtering threshold applied to expected errors. By default, all expected errors. Mar recommends using c(1,1). Here, if there is maxEE expected errors, its okay. If more, throw away sequence.
  • trimLeft = trim certain number of base pairs on start of each read
  • truncQ = truncate reads at the first instance of a quality score less than or equal to selected number. Chose 2
  • rm.phix = remove phi x
  • compress = make filtered files .gzipped
  • multithread = multithread

806 R primer is GGACTACHVGGGTWTCTAAT (21 bp)

#Assign a vector to filtered reads
#Trim out poor bases
#Write out filtered fastq files
filtered_reads <-
  filterAndTrim(fwd = reverse_reads, filt = filtered_reverse_reads,
              truncLen = 245, trimLeft = 21,
              maxN = 0, maxEE = 2,truncQ = 2, rm.phix = TRUE,
              compress = TRUE, multithread = 6)

Trimmed Quality Plots

# Plot the 12 random samples after QC
reverse_filteredQual_plot_12 <- 
  plotQualityProfile(filtered_reverse_reads[random_samples]) + 
  labs(title = "Trimmed Reverse Read Quality")

# Show plots
reverse_filteredQual_plot_12

Aggregated Trimmed Plots

# Aggregate reverse reads
reverse_postQC_plot <- 
  plotQualityProfile(filtered_reverse_reads, aggregate = TRUE) + 
  labs(title = "Reverse Post-QC")

# Show the plot
reverse_postQC_plot

Stats on read output from filterAndTrim

#Make output into dataframe
filtered_df <- as.data.frame(filtered_reads)
head(filtered_df)
##                        reads.in reads.out
## SRR17060816_2.fastq.gz   285558    180782
## SRR17060817_2.fastq.gz   676817    316195
## SRR17060818_2.fastq.gz   591364    316006
## SRR17060819_2.fastq.gz   379452    202870
## SRR17060820_2.fastq.gz   570270    298207
## SRR17060821_2.fastq.gz   556682    300096
# calculate some stats
filtered_df %>%
  reframe(median_reads_in = median(reads.in),
          median_reads_out = median(reads.out),
          median_percent_retained = (median(reads.out)/median(reads.in)))
##   median_reads_in median_reads_out median_percent_retained
## 1        294748.5         155756.5               0.5284387

This is an improvement over other filter and trims (with forward and reverse reads. We retained about 52% of the reads.)

Error Modeling

Note every sequencing run needs to be run separately! The error model MUST be run separately on each illumina dataset. If you’d like to combine the datasets from multiple sequencing runs, you’ll need to do the exact same filterAndTrim() step AND, very importantly, you’ll need to have the same primer and ASV length expected by the output.

Infer error rates for all possible transitions within purines and pyrimidines (A<>G or C<>T) and transversions between all purine and pyrimidine combinations.

Error model is learned by alternating estimation of the error rates and inference of sample composition until they converge.

  1. Starts with the assumption that the error rates are the maximum (takes the most abundant sequence (“center”) and assumes it’s the only sequence not caused by errors).
  2. Compares the other sequences to the most abundant sequence.
  3. Uses at most 108 nucleotides for the error estimation.
  4. Uses parametric error estimation function of loess fit on the observed error rates.
#Reverse reads
error_reverse_reads <-
  learnErrors(filtered_reverse_reads, multithread = 6)
## 111322848 total bases in 496977 reads from 2 samples will be used for learning the error rates.
#Plot reverse reads errors
reverse_error_plot <-
  plotErrors(error_reverse_reads, nominalQ = TRUE) +
    labs(title = "Reverse Read Error Model")

#Show plot
reverse_error_plot
## Warning in scale_y_log10(): log-10 transformation introduced infinite values.

It looks like the points follow the black lines pretty well.

  • The error rates for each possible transition (A→C, A→G, …) are shown in the plot above.

Details of the plot: - Points: The observed error rates for each consensus quality score.
- Black line: Estimated error rates after convergence of the machine-learning algorithm.
- Red line: The error rates expected under the nominal definition of the Q-score.

Similar to what is mentioned in the dada2 tutorial: the estimated error rates (black line) are a “reasonably good” fit to the observed rates (points), and the error rates drop with increased quality as expected. We can now infer ASVs!

Infer ASVs

An important note: This process occurs separately on forward and reverse reads! This is quite a different approach from how OTUs are identified in Mothur and also from UCHIME, oligotyping, and other OTU, MED, and ASV approaches.

#Infer reverse ASVs
dada_reverse <- dada(filtered_reverse_reads, 
                     err = error_reverse_reads, 
                     multithread = 6)
## Sample 1 - 180782 reads in 74958 unique sequences.
## Sample 2 - 316195 reads in 82923 unique sequences.
## Sample 3 - 316006 reads in 87472 unique sequences.
## Sample 4 - 202870 reads in 93346 unique sequences.
## Sample 5 - 298207 reads in 89269 unique sequences.
## Sample 6 - 300096 reads in 87821 unique sequences.
## Sample 7 - 94957 reads in 50259 unique sequences.
## Sample 8 - 139760 reads in 88647 unique sequences.
## Sample 9 - 206345 reads in 73576 unique sequences.
## Sample 10 - 75785 reads in 45955 unique sequences.
## Sample 11 - 68357 reads in 41724 unique sequences.
## Sample 12 - 57632 reads in 30944 unique sequences.
## Sample 13 - 79869 reads in 46806 unique sequences.
## Sample 14 - 2674 reads in 2191 unique sequences.
## Sample 15 - 19745 reads in 10488 unique sequences.
## Sample 16 - 300732 reads in 122848 unique sequences.
## Sample 17 - 156352 reads in 55908 unique sequences.
## Sample 18 - 293178 reads in 101504 unique sequences.
## Sample 19 - 289316 reads in 132351 unique sequences.
## Sample 20 - 267030 reads in 117493 unique sequences.
## Sample 21 - 286913 reads in 86289 unique sequences.
## Sample 22 - 169592 reads in 89288 unique sequences.
## Sample 23 - 271815 reads in 89396 unique sequences.
## Sample 24 - 2969 reads in 2082 unique sequences.
## Sample 25 - 113512 reads in 62398 unique sequences.
## Sample 26 - 86355 reads in 44348 unique sequences.
## Sample 27 - 79405 reads in 35503 unique sequences.
## Sample 28 - 80227 reads in 40627 unique sequences.
## Sample 29 - 82394 reads in 55600 unique sequences.
## Sample 30 - 5584 reads in 3954 unique sequences.
## Sample 31 - 229742 reads in 96275 unique sequences.
## Sample 32 - 155161 reads in 68238 unique sequences.
#Inspect
dada_reverse[1]
## $SRR17060816_R2_filtered.fastq.gz
## dada-class: object describing DADA2 denoising results
## 398 sequence variants were inferred from 74958 input unique sequences.
## Key parameters: OMEGA_A = 1e-40, OMEGA_C = 1e-40, BAND_SIZE = 16
dada_reverse[12]
## $SRR17060827_R2_filtered.fastq.gz
## dada-class: object describing DADA2 denoising results
## 73 sequence variants were inferred from 30944 input unique sequences.
## Key parameters: OMEGA_A = 1e-40, OMEGA_C = 1e-40, BAND_SIZE = 16

Create Raw ASV Count Table

# Create the ASV Count Table 
raw_ASV_table <- makeSequenceTable(dada_reverse)

# Write out the file to data/01_DADA2


# Check the type and dimensions of the data
dim(raw_ASV_table)
## [1]   32 4873
class(raw_ASV_table)
## [1] "matrix" "array"
typeof(raw_ASV_table)
## [1] "integer"
# Inspect the distribution of sequence lengths of all ASVs in dataset 
table(nchar(getSequences(raw_ASV_table)))
## 
##  224 
## 4873
# All reads are 224 bp

# Inspect the distribution of sequence lengths of all ASVs in dataset 
data.frame(Seq_Length = nchar(getSequences(raw_ASV_table))) %>%
  ggplot(aes(x = Seq_Length )) + 
  geom_histogram() + 
  labs(title = "Raw distribution of ASV length")
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

Taking into account the lower, zoomed-in plot. Do we want to remove those extra ASVs?

Remove Chimeras

Sometimes chimeras arise in our workflow.

Chimeric sequences are artificial sequences formed by the combination of two or more distinct biological sequences. These chimeric sequences can arise during the polymerase chain reaction (PCR) amplification step of the 16S rRNA gene, where fragments from different templates can be erroneously joined together.

Chimera removal is an essential step in the analysis of 16S sequencing data to improve the accuracy of downstream analyses, such as taxonomic assignment and diversity assessment. It helps to avoid the inclusion of misleading or spurious sequences that could lead to incorrect biological interpretations.

# Remove the chimeras in the raw ASV table
noChimeras_ASV_table <- removeBimeraDenovo(raw_ASV_table, 
                                           method="consensus", 
                                           multithread=6, verbose=TRUE)
## Identified 416 bimeras out of 4873 input sequences.
# Check the dimensions
dim(noChimeras_ASV_table)
## [1]   32 4457
# What proportion is left of the sequences? 
sum(noChimeras_ASV_table)/sum(raw_ASV_table)
## [1] 0.9864887
# Plot it 
data.frame(Seq_Length_NoChim = nchar(getSequences(noChimeras_ASV_table))) %>%
  ggplot(aes(x = Seq_Length_NoChim )) + 
  geom_histogram()+ 
  labs(title = "Trimmed + Chimera Removal distribution of ASV length")
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

# Note the difference in the peak at 249, which is now BELOW 3000

Track the read counts

Here, we will look at the number of reads that were lost in the filtering, denoising, merging, and chimera removal.

# A little function to identify number seqs 
getN <- function(x) sum(getUniques(x))

# Make the table to track the seqs 
track <- cbind(filtered_reads,
               sapply(dada_reverse, getN),
               rowSums(noChimeras_ASV_table))

head(track)
##                        reads.in reads.out              
## SRR17060816_2.fastq.gz   285558    180782 179749 175977
## SRR17060817_2.fastq.gz   676817    316195 315529 314763
## SRR17060818_2.fastq.gz   591364    316006 315414 313288
## SRR17060819_2.fastq.gz   379452    202870 202117 200174
## SRR17060820_2.fastq.gz   570270    298207 297674 297140
## SRR17060821_2.fastq.gz   556682    300096 299442 299313
# Update column names to be more informative (most are missing at the moment!)
colnames(track) <- c("input", "filtered", "denoisedR", "nochim")
rownames(track) <- samples

# Generate a dataframe to track the reads through our DADA2 pipeline
track_counts_df <- 
  track %>%
  # make it a dataframe
  as.data.frame() %>%
  rownames_to_column(var = "names") %>%
  mutate(perc_reads_retained = 100 * nochim / input)

# Visualize it in table format 
DT::datatable(track_counts_df)
# Plot it!
track_counts_df %>%
  pivot_longer(input:nochim, names_to = "read_type", values_to = "num_reads") %>%
  mutate(read_type = fct_relevel(read_type, 
                                 "input", "filtered", "denoisedR", "nochim")) %>%
  ggplot(aes(x = read_type, y = num_reads, fill = read_type)) + 
  geom_line(aes(group = names), color = "grey") + 
  geom_point(shape = 21, size = 3, alpha = 0.8) + 
  scale_fill_brewer(palette = "Spectral") + 
  labs(x = "Filtering Step", y = "Number of Sequences") + 
  theme_bw()

Assign Taxonomy

Here, we will use the silva database version 138!

# The next line took 2 mins to run
taxa_train <- 
  assignTaxonomy(noChimeras_ASV_table, 
                 "/workdir/in_class_data/taxonomy/silva_nr99_v138.1_train_set.fa.gz", 
                 multithread=6)

# the next line took 3 minutes 
taxa_addSpecies <- 
  addSpecies(taxa_train, 
             "/workdir/in_class_data/taxonomy/silva_species_assignment_v138.1.fa.gz")

# Inspect the taxonomy 
taxa_print <- taxa_addSpecies # Removing sequence rownames for display only
rownames(taxa_print) <- NULL
#View(taxa_print)

Prepare the data for export!

1. ASV Table

Below, we will prepare the following:

  1. Two ASV Count tables:
    1. With ASV seqs: ASV headers include the entire ASV sequence ~250bps.
    2. with ASV names: This includes re-written and shortened headers like ASV_1, ASV_2, etc, which will match the names in our fasta file below.
  2. ASV_fastas: A fasta file that we can use to build a tree for phylogenetic analyses (e.g. phylogenetic alpha diversity metrics or UNIFRAC dissimilarty).

Finalize ASV Count Tables

########### 2. COUNT TABLE ###############
############## Modify the ASV names and then save a fasta file!  ############## 
# Give headers more manageable names
# First pull the ASV sequences
asv_seqs <- colnames(noChimeras_ASV_table)
asv_seqs[1:5]
## [1] "TGTTTGCTCCCCACGCTTTCGTACCTCAGCGTCAGTGTCAGACCAGAGTGTCGCCTTCGCCACTGGTGTTCCTTCCTATATCTACGCATTTCACCGCTACACAGGAAATTCCACACTCCTCTTCCGCACTCGAGCCTCTCAGTTTTGGATGCCGTTCCCAGGTTGAGCCCGGGGCTTTCACATCCAACTTAACAGGCCGCCTACGCACGCTTTACGCCCAGTAA"
## [2] "TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTTGGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTGGAGTTGAGCCCCAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA"
## [3] "TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTCAGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTAGAGTTGAGCCCTAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA"
## [4] "CGCATCTGAGCGTCAGTCTTTGTCCAGGGGGCCGCCTTCGCCACCGGTATTCCTTCAGATCTCTACGCATTTCACCGCTACACCTGAAATTCTACCCCCCTCTACAAGACTCTAGCTTGCCAGTTCAAAATGCGATTCCCAGGTTAAGCCCGGGGCTTTCACATCTTGCTTAACAAACCGCCTGCATGCGCTTTACGCCCAGTAATTCCGATTAACGCTCGCAC"
## [5] "TGTTTGCTCCCCATGCTTTCGTACCTCAGCGTCAGTATTAGGCCAGATGGCTGCCTTCGCCATCGGTATTCCTCCAGATCTCTACGCATTTCACCGCTACACCTGGAATTCTACCATCCTCTCCCATACTCTAGCTTCCCAGTATCGAATGCAATTCCTAAGTTAAGCTCAGGGATTTCACATCCGACTTAAAAAGCCGCCTACGCACGCTTTACGCCCAGTAA"
# make headers for our ASV seq fasta file, which will be our asv names
asv_headers <- vector(dim(noChimeras_ASV_table)[2], mode = "character")
asv_headers[1:5]
## [1] "" "" "" "" ""
# loop through vector and fill it in with ASV names 
for (i in 1:dim(noChimeras_ASV_table)[2]) {
  asv_headers[i] <- paste(">ASV", i, sep = "_")
}

# intitution check
asv_headers[1:5]
## [1] ">ASV_1" ">ASV_2" ">ASV_3" ">ASV_4" ">ASV_5"
##### Rename ASVs in table then write out our ASV fasta file! 
#View(noChimeras_ASV_table)
asv_tab <- t(noChimeras_ASV_table)
#View(asv_tab)

## Rename our asvs! 
row.names(asv_tab) <- sub(">", "", asv_headers)
#View(asv_tab)

2. Taxonomy Table

# Inspect the taxonomy table
#View(taxa_addSpecies)

##### Prepare tax table 
# Add the ASV sequences from the rownames to a column 
new_tax_tab <- 
  taxa_addSpecies%>%
  as.data.frame() %>%
  rownames_to_column(var = "ASVseqs") 
head(new_tax_tab)
##                                                                                                                                                                                                                            ASVseqs
## 1 TGTTTGCTCCCCACGCTTTCGTACCTCAGCGTCAGTGTCAGACCAGAGTGTCGCCTTCGCCACTGGTGTTCCTTCCTATATCTACGCATTTCACCGCTACACAGGAAATTCCACACTCCTCTTCCGCACTCGAGCCTCTCAGTTTTGGATGCCGTTCCCAGGTTGAGCCCGGGGCTTTCACATCCAACTTAACAGGCCGCCTACGCACGCTTTACGCCCAGTAA
## 2 TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTTGGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTGGAGTTGAGCCCCAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA
## 3 TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTCAGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTAGAGTTGAGCCCTAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA
## 4 CGCATCTGAGCGTCAGTCTTTGTCCAGGGGGCCGCCTTCGCCACCGGTATTCCTTCAGATCTCTACGCATTTCACCGCTACACCTGAAATTCTACCCCCCTCTACAAGACTCTAGCTTGCCAGTTCAAAATGCGATTCCCAGGTTAAGCCCGGGGCTTTCACATCTTGCTTAACAAACCGCCTGCATGCGCTTTACGCCCAGTAATTCCGATTAACGCTCGCAC
## 5 TGTTTGCTCCCCATGCTTTCGTACCTCAGCGTCAGTATTAGGCCAGATGGCTGCCTTCGCCATCGGTATTCCTCCAGATCTCTACGCATTTCACCGCTACACCTGGAATTCTACCATCCTCTCCCATACTCTAGCTTCCCAGTATCGAATGCAATTCCTAAGTTAAGCTCAGGGATTTCACATCCGACTTAAAAAGCCGCCTACGCACGCTTTACGCCCAGTAA
## 6 CGTGCCTCAGCGTCAGTTGTATGTTAGTCAGCTGCCTTCGCAATCGGAGTTCTTCGTTATATCTATGCATTTCACCGCTACACAACGAATTCCGCCAACTTCATTTACACTCAAGTCTCCCAGTTTCAATGCCAATTTTCCGGTTGAGCCGAAAACTTTCAACGCTGACTTAAGAGACCGCCTACGCACCCTTTAAACCCAATAAATCCGGATAACGCTCGGAT
##    Kingdom Phylum Class Order Family Genus Species
## 1 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## 2 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## 3 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## 4 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## 5 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## 6 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
# intution check 
stopifnot(new_tax_tab$ASVseqs == colnames(noChimeras_ASV_table))

# Now let's add the ASV names 
rownames(new_tax_tab) <- rownames(asv_tab)
head(new_tax_tab)
##                                                                                                                                                                                                                                ASVseqs
## ASV_1 TGTTTGCTCCCCACGCTTTCGTACCTCAGCGTCAGTGTCAGACCAGAGTGTCGCCTTCGCCACTGGTGTTCCTTCCTATATCTACGCATTTCACCGCTACACAGGAAATTCCACACTCCTCTTCCGCACTCGAGCCTCTCAGTTTTGGATGCCGTTCCCAGGTTGAGCCCGGGGCTTTCACATCCAACTTAACAGGCCGCCTACGCACGCTTTACGCCCAGTAA
## ASV_2 TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTTGGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTGGAGTTGAGCCCCAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA
## ASV_3 TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTCAGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTAGAGTTGAGCCCTAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA
## ASV_4 CGCATCTGAGCGTCAGTCTTTGTCCAGGGGGCCGCCTTCGCCACCGGTATTCCTTCAGATCTCTACGCATTTCACCGCTACACCTGAAATTCTACCCCCCTCTACAAGACTCTAGCTTGCCAGTTCAAAATGCGATTCCCAGGTTAAGCCCGGGGCTTTCACATCTTGCTTAACAAACCGCCTGCATGCGCTTTACGCCCAGTAATTCCGATTAACGCTCGCAC
## ASV_5 TGTTTGCTCCCCATGCTTTCGTACCTCAGCGTCAGTATTAGGCCAGATGGCTGCCTTCGCCATCGGTATTCCTCCAGATCTCTACGCATTTCACCGCTACACCTGGAATTCTACCATCCTCTCCCATACTCTAGCTTCCCAGTATCGAATGCAATTCCTAAGTTAAGCTCAGGGATTTCACATCCGACTTAAAAAGCCGCCTACGCACGCTTTACGCCCAGTAA
## ASV_6 CGTGCCTCAGCGTCAGTTGTATGTTAGTCAGCTGCCTTCGCAATCGGAGTTCTTCGTTATATCTATGCATTTCACCGCTACACAACGAATTCCGCCAACTTCATTTACACTCAAGTCTCCCAGTTTCAATGCCAATTTTCCGGTTGAGCCGAAAACTTTCAACGCTGACTTAAGAGACCGCCTACGCACCCTTTAAACCCAATAAATCCGGATAACGCTCGGAT
##        Kingdom Phylum Class Order Family Genus Species
## ASV_1 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## ASV_2 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## ASV_3 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## ASV_4 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## ASV_5 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
## ASV_6 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA>
### Final prep of tax table. Add new column with ASV names 
asv_tax <- 
  new_tax_tab %>%
  # add rownames from count table for phyloseq handoff
  mutate(ASV = rownames(asv_tab)) %>%
  # Resort the columns with select
  dplyr::select(Kingdom, Phylum, Class, Order, Family, Genus, Species, ASV, ASVseqs)

head(asv_tax)
##        Kingdom Phylum Class Order Family Genus Species   ASV
## ASV_1 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA> ASV_1
## ASV_2 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA> ASV_2
## ASV_3 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA> ASV_3
## ASV_4 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA> ASV_4
## ASV_5 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA> ASV_5
## ASV_6 Bacteria   <NA>  <NA>  <NA>   <NA>  <NA>    <NA> ASV_6
##                                                                                                                                                                                                                                ASVseqs
## ASV_1 TGTTTGCTCCCCACGCTTTCGTACCTCAGCGTCAGTGTCAGACCAGAGTGTCGCCTTCGCCACTGGTGTTCCTTCCTATATCTACGCATTTCACCGCTACACAGGAAATTCCACACTCCTCTTCCGCACTCGAGCCTCTCAGTTTTGGATGCCGTTCCCAGGTTGAGCCCGGGGCTTTCACATCCAACTTAACAGGCCGCCTACGCACGCTTTACGCCCAGTAA
## ASV_2 TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTTGGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTGGAGTTGAGCCCCAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA
## ASV_3 TATTTGATCCCCACACTTTCGCGCCTCAGCGTCAATCTCAGCATAGACAACTGCCTTCGCCTTTGGTGTTCCTTCCCATATCTATGCATTCCACCGCTACACGGGAAATTCCGTTGCCTTCCACCAGATTCTAGTCAACCGGTTCTGAATGCCCTTCTAGAGTTGAGCCCTAGTCTTTAACACTCAGCCTAATTAACCGCCTACACGCCCTTTACGCCCAGTAA
## ASV_4 CGCATCTGAGCGTCAGTCTTTGTCCAGGGGGCCGCCTTCGCCACCGGTATTCCTTCAGATCTCTACGCATTTCACCGCTACACCTGAAATTCTACCCCCCTCTACAAGACTCTAGCTTGCCAGTTCAAAATGCGATTCCCAGGTTAAGCCCGGGGCTTTCACATCTTGCTTAACAAACCGCCTGCATGCGCTTTACGCCCAGTAATTCCGATTAACGCTCGCAC
## ASV_5 TGTTTGCTCCCCATGCTTTCGTACCTCAGCGTCAGTATTAGGCCAGATGGCTGCCTTCGCCATCGGTATTCCTCCAGATCTCTACGCATTTCACCGCTACACCTGGAATTCTACCATCCTCTCCCATACTCTAGCTTCCCAGTATCGAATGCAATTCCTAAGTTAAGCTCAGGGATTTCACATCCGACTTAAAAAGCCGCCTACGCACGCTTTACGCCCAGTAA
## ASV_6 CGTGCCTCAGCGTCAGTTGTATGTTAGTCAGCTGCCTTCGCAATCGGAGTTCTTCGTTATATCTATGCATTTCACCGCTACACAACGAATTCCGCCAACTTCATTTACACTCAAGTCTCCCAGTTTCAATGCCAATTTTCCGGTTGAGCCGAAAACTTTCAACGCTGACTTAAGAGACCGCCTACGCACCCTTTAAACCCAATAAATCCGGATAACGCTCGGAT
# Intution check
stopifnot(asv_tax$ASV == rownames(asv_tax), rownames(asv_tax) == rownames(asv_tab))

Write 01_DADA2 files

Now, we will write the files! We will write the following to the data/01_DADA2/ folder. We will save both as files that could be submitted as supplements AND as .RData objects for easy loading into the next steps into R.:

  1. ASV_counts.tsv: ASV count table that has ASV names that are re-written and shortened headers like ASV_1, ASV_2, etc, which will match the names in our fasta file below. This will also be saved as data/01_DADA2/ASV_counts.RData.
  2. ASV_counts_withSeqNames.tsv: This is generated with the data object in this file known as noChimeras_ASV_table. ASV headers include the entire ASV sequence ~250bps. In addition, we will save this as a .RData object as data/01_DADA2/noChimeras_ASV_table.RData as we will use this data in analysis/02_Taxonomic_Assignment.Rmd to assign the taxonomy from the sequence headers.
  3. ASVs.fasta: A fasta file output of the ASV names from ASV_counts.tsv and the sequences from the ASVs in ASV_counts_withSeqNames.tsv. A fasta file that we can use to build a tree for phylogenetic analyses (e.g. phylogenetic alpha diversity metrics or UNIFRAC dissimilarty).
  4. We will also make a copy of ASVs.fasta in data/02_TaxAss_FreshTrain/ to be used for the taxonomy classification in the next step in the workflow.
  5. Write out the taxonomy table
  6. track_read_counts.RData: To track how many reads we lost throughout our workflow that could be used and plotted later. We will add this to the metadata in analysis/02_Taxonomic_Assignment.Rmd.
# FIRST, we will save our output as regular files, which will be useful later on. 
# Save to regular .tsv file 
# Write BOTH the modified and unmodified ASV tables to a file!
# Write count table with ASV numbered names (e.g. ASV_1, ASV_2, etc)
write.table(asv_tab, "data/01_DADA2/ASV_counts.tsv", sep = "\t", quote = FALSE, col.names = NA)
# Write count table with ASV sequence names
write.table(noChimeras_ASV_table, "data/01_DADA2/ASV_counts_withSeqNames.tsv", sep = "\t", quote = FALSE, col.names = NA)
# Write out the fasta file for reference later on for what seq matches what ASV
asv_fasta <- c(rbind(asv_headers, asv_seqs))
# Save to a file!
write(asv_fasta, "data/01_DADA2/ASVs.fasta")


# SECOND, let's save the taxonomy tables 
# Write the table 
write.table(asv_tax, "data/01_DADA2/ASV_taxonomy.tsv", sep = "\t", quote = FALSE, col.names = NA)


# THIRD, let's save to a RData object 
# Each of these files will be used in the analysis/02_Taxonomic_Assignment
# RData objects are for easy loading :) 
save(noChimeras_ASV_table, file = "data/01_DADA2/noChimeras_ASV_table.RData")
save(asv_tab, file = "data/01_DADA2/ASV_counts.RData")
# And save the track_counts_df a R object, which we will merge with metadata information in the next step of the analysis in nalysis/02_Taxonomic_Assignment. 
save(track_counts_df, file = "data/01_DADA2/track_read_counts.RData")

##Session information

#Ensure reproducibility
devtools::session_info()
## ─ Session info ───────────────────────────────────────────────────────────────
##  setting  value
##  version  R version 4.3.2 (2023-10-31)
##  os       Rocky Linux 9.0 (Blue Onyx)
##  system   x86_64, linux-gnu
##  ui       X11
##  language (EN)
##  collate  en_US.UTF-8
##  ctype    en_US.UTF-8
##  tz       America/New_York
##  date     2024-04-16
##  pandoc   3.1.1 @ /usr/lib/rstudio-server/bin/quarto/bin/tools/ (via rmarkdown)
## 
## ─ Packages ───────────────────────────────────────────────────────────────────
##  package              * version    date (UTC) lib source
##  abind                  1.4-5      2016-07-21 [2] CRAN (R 4.3.2)
##  ade4                   1.7-22     2023-02-06 [1] CRAN (R 4.3.2)
##  ape                    5.8        2024-04-11 [1] CRAN (R 4.3.2)
##  Biobase                2.62.0     2023-10-24 [2] Bioconductor
##  BiocGenerics           0.48.1     2023-11-01 [2] Bioconductor
##  BiocParallel           1.36.0     2023-10-24 [2] Bioconductor
##  biomformat             1.30.0     2023-10-24 [1] Bioconductor
##  Biostrings             2.70.1     2023-10-25 [2] Bioconductor
##  bitops                 1.0-7      2021-04-24 [2] CRAN (R 4.3.2)
##  bslib                  0.5.1      2023-08-11 [2] CRAN (R 4.3.2)
##  cachem                 1.0.8      2023-05-01 [2] CRAN (R 4.3.2)
##  callr                  3.7.3      2022-11-02 [2] CRAN (R 4.3.2)
##  cli                    3.6.1      2023-03-23 [2] CRAN (R 4.3.2)
##  cluster                2.1.4      2022-08-22 [2] CRAN (R 4.3.2)
##  codetools              0.2-19     2023-02-01 [2] CRAN (R 4.3.2)
##  colorspace             2.1-0      2023-01-23 [2] CRAN (R 4.3.2)
##  crayon                 1.5.2      2022-09-29 [2] CRAN (R 4.3.2)
##  crosstalk              1.2.0      2021-11-04 [2] CRAN (R 4.3.2)
##  dada2                * 1.30.0     2023-10-24 [1] Bioconductor
##  data.table             1.14.8     2023-02-17 [2] CRAN (R 4.3.2)
##  DelayedArray           0.28.0     2023-10-24 [2] Bioconductor
##  deldir                 1.0-9      2023-05-17 [2] CRAN (R 4.3.2)
##  devtools             * 2.4.4      2022-07-20 [2] CRAN (R 4.2.1)
##  digest                 0.6.33     2023-07-07 [2] CRAN (R 4.3.2)
##  dplyr                * 1.1.3      2023-09-03 [2] CRAN (R 4.3.2)
##  DT                   * 0.32       2024-02-19 [1] CRAN (R 4.3.2)
##  ellipsis               0.3.2      2021-04-29 [2] CRAN (R 4.3.2)
##  evaluate               0.23       2023-11-01 [2] CRAN (R 4.3.2)
##  fansi                  1.0.5      2023-10-08 [2] CRAN (R 4.3.2)
##  farver                 2.1.1      2022-07-06 [2] CRAN (R 4.3.2)
##  fastmap                1.1.1      2023-02-24 [2] CRAN (R 4.3.2)
##  forcats              * 1.0.0      2023-01-29 [1] CRAN (R 4.3.2)
##  foreach                1.5.2      2022-02-02 [2] CRAN (R 4.3.2)
##  fs                     1.6.3      2023-07-20 [2] CRAN (R 4.3.2)
##  generics               0.1.3      2022-07-05 [2] CRAN (R 4.3.2)
##  GenomeInfoDb           1.38.0     2023-10-24 [2] Bioconductor
##  GenomeInfoDbData       1.2.11     2023-11-07 [2] Bioconductor
##  GenomicAlignments      1.38.0     2023-10-24 [2] Bioconductor
##  GenomicRanges          1.54.1     2023-10-29 [2] Bioconductor
##  ggplot2              * 3.5.0      2024-02-23 [2] CRAN (R 4.3.2)
##  glue                   1.6.2      2022-02-24 [2] CRAN (R 4.3.2)
##  gtable                 0.3.4      2023-08-21 [2] CRAN (R 4.3.2)
##  highr                  0.10       2022-12-22 [2] CRAN (R 4.3.2)
##  hms                    1.1.3      2023-03-21 [1] CRAN (R 4.3.2)
##  htmltools              0.5.7      2023-11-03 [2] CRAN (R 4.3.2)
##  htmlwidgets            1.6.2      2023-03-17 [2] CRAN (R 4.3.2)
##  httpuv                 1.6.12     2023-10-23 [2] CRAN (R 4.3.2)
##  hwriter                1.3.2.1    2022-04-08 [1] CRAN (R 4.3.2)
##  igraph                 1.5.1      2023-08-10 [2] CRAN (R 4.3.2)
##  interp                 1.1-6      2024-01-26 [1] CRAN (R 4.3.2)
##  IRanges                2.36.0     2023-10-24 [2] Bioconductor
##  iterators              1.0.14     2022-02-05 [2] CRAN (R 4.3.2)
##  jpeg                   0.1-10     2022-11-29 [1] CRAN (R 4.3.2)
##  jquerylib              0.1.4      2021-04-26 [2] CRAN (R 4.3.2)
##  jsonlite               1.8.7      2023-06-29 [2] CRAN (R 4.3.2)
##  knitr                  1.45       2023-10-30 [2] CRAN (R 4.3.2)
##  labeling               0.4.3      2023-08-29 [2] CRAN (R 4.3.2)
##  later                  1.3.1      2023-05-02 [2] CRAN (R 4.3.2)
##  lattice                0.21-9     2023-10-01 [2] CRAN (R 4.3.2)
##  latticeExtra           0.6-30     2022-07-04 [1] CRAN (R 4.3.2)
##  lifecycle              1.0.3      2022-10-07 [2] CRAN (R 4.3.2)
##  lubridate            * 1.9.3      2023-09-27 [1] CRAN (R 4.3.2)
##  magrittr               2.0.3      2022-03-30 [2] CRAN (R 4.3.2)
##  MASS                   7.3-60     2023-05-04 [2] CRAN (R 4.3.2)
##  Matrix                 1.6-1.1    2023-09-18 [2] CRAN (R 4.3.2)
##  MatrixGenerics         1.14.0     2023-10-24 [2] Bioconductor
##  matrixStats            1.1.0      2023-11-07 [2] CRAN (R 4.3.2)
##  memoise                2.0.1      2021-11-26 [2] CRAN (R 4.3.2)
##  mgcv                   1.9-0      2023-07-11 [2] CRAN (R 4.3.2)
##  mime                   0.12       2021-09-28 [2] CRAN (R 4.3.2)
##  miniUI                 0.1.1.1    2018-05-18 [2] CRAN (R 4.3.2)
##  multtest               2.58.0     2023-10-24 [1] Bioconductor
##  munsell                0.5.0      2018-06-12 [2] CRAN (R 4.3.2)
##  nlme                   3.1-163    2023-08-09 [2] CRAN (R 4.3.2)
##  pacman                 0.5.1      2019-03-11 [1] CRAN (R 4.3.2)
##  patchwork            * 1.2.0.9000 2024-03-12 [1] Github (thomasp85/patchwork@d943757)
##  permute                0.9-7      2022-01-27 [1] CRAN (R 4.3.2)
##  phyloseq             * 1.41.1     2024-03-09 [1] Github (joey711/phyloseq@c260561)
##  pillar                 1.9.0      2023-03-22 [2] CRAN (R 4.3.2)
##  pkgbuild               1.4.2      2023-06-26 [2] CRAN (R 4.3.2)
##  pkgconfig              2.0.3      2019-09-22 [2] CRAN (R 4.3.2)
##  pkgload                1.3.3      2023-09-22 [2] CRAN (R 4.3.2)
##  plyr                   1.8.9      2023-10-02 [2] CRAN (R 4.3.2)
##  png                    0.1-8      2022-11-29 [2] CRAN (R 4.3.2)
##  prettyunits            1.2.0      2023-09-24 [2] CRAN (R 4.3.2)
##  processx               3.8.2      2023-06-30 [2] CRAN (R 4.3.2)
##  profvis                0.3.8      2023-05-02 [2] CRAN (R 4.3.2)
##  promises               1.2.1      2023-08-10 [2] CRAN (R 4.3.2)
##  ps                     1.7.5      2023-04-18 [2] CRAN (R 4.3.2)
##  purrr                * 1.0.2      2023-08-10 [2] CRAN (R 4.3.2)
##  R6                     2.5.1      2021-08-19 [2] CRAN (R 4.3.2)
##  RColorBrewer           1.1-3      2022-04-03 [2] CRAN (R 4.3.2)
##  Rcpp                 * 1.0.11     2023-07-06 [2] CRAN (R 4.3.2)
##  RcppParallel           5.1.7      2023-02-27 [2] CRAN (R 4.3.2)
##  RCurl                  1.98-1.13  2023-11-02 [2] CRAN (R 4.3.2)
##  readr                * 2.1.5      2024-01-10 [1] CRAN (R 4.3.2)
##  remotes                2.4.2.1    2023-07-18 [2] CRAN (R 4.3.2)
##  reshape2               1.4.4      2020-04-09 [2] CRAN (R 4.3.2)
##  rhdf5                  2.46.1     2023-11-29 [1] Bioconductor 3.18 (R 4.3.2)
##  rhdf5filters           1.14.1     2023-11-06 [1] Bioconductor
##  Rhdf5lib               1.24.2     2024-02-07 [1] Bioconductor 3.18 (R 4.3.2)
##  rlang                  1.1.2      2023-11-04 [2] CRAN (R 4.3.2)
##  rmarkdown              2.25       2023-09-18 [2] CRAN (R 4.3.2)
##  Rsamtools              2.18.0     2023-10-24 [2] Bioconductor
##  rstudioapi             0.15.0     2023-07-07 [2] CRAN (R 4.3.2)
##  S4Arrays               1.2.0      2023-10-24 [2] Bioconductor
##  S4Vectors              0.40.1     2023-10-26 [2] Bioconductor
##  sass                   0.4.7      2023-07-15 [2] CRAN (R 4.3.2)
##  scales                 1.3.0      2023-11-28 [2] CRAN (R 4.3.2)
##  sessioninfo            1.2.2      2021-12-06 [2] CRAN (R 4.3.2)
##  shiny                  1.7.5.1    2023-10-14 [2] CRAN (R 4.3.2)
##  ShortRead              1.60.0     2023-10-24 [1] Bioconductor
##  SparseArray            1.2.1      2023-11-05 [2] Bioconductor
##  stringi                1.7.12     2023-01-11 [2] CRAN (R 4.3.2)
##  stringr              * 1.5.0      2022-12-02 [2] CRAN (R 4.3.2)
##  SummarizedExperiment   1.32.0     2023-10-24 [2] Bioconductor
##  survival               3.5-7      2023-08-14 [2] CRAN (R 4.3.2)
##  tibble               * 3.2.1      2023-03-20 [2] CRAN (R 4.3.2)
##  tidyr                * 1.3.0      2023-01-24 [2] CRAN (R 4.3.2)
##  tidyselect             1.2.0      2022-10-10 [2] CRAN (R 4.3.2)
##  tidyverse            * 2.0.0      2023-02-22 [1] CRAN (R 4.3.2)
##  timechange             0.3.0      2024-01-18 [1] CRAN (R 4.3.2)
##  tzdb                   0.4.0      2023-05-12 [1] CRAN (R 4.3.2)
##  urlchecker             1.0.1      2021-11-30 [2] CRAN (R 4.3.2)
##  usethis              * 2.2.2      2023-07-06 [2] CRAN (R 4.3.2)
##  utf8                   1.2.4      2023-10-22 [2] CRAN (R 4.3.2)
##  vctrs                  0.6.4      2023-10-12 [2] CRAN (R 4.3.2)
##  vegan                  2.6-4      2022-10-11 [1] CRAN (R 4.3.2)
##  withr                  2.5.2      2023-10-30 [2] CRAN (R 4.3.2)
##  xfun                   0.41       2023-11-01 [2] CRAN (R 4.3.2)
##  xtable                 1.8-4      2019-04-21 [2] CRAN (R 4.3.2)
##  XVector                0.42.0     2023-10-24 [2] Bioconductor
##  yaml                   2.3.7      2023-01-23 [2] CRAN (R 4.3.2)
##  zlibbioc               1.48.0     2023-10-24 [2] Bioconductor
## 
##  [1] /home/cab565/R/x86_64-pc-linux-gnu-library/4.3
##  [2] /programs/R-4.3.2/library
## 
## ──────────────────────────────────────────────────────────────────────────────